#! /usr/bin/python

import subprocess
import re
import time
import random
from gethostinList import getHost


def genericGetPage(postCmd,suffix,maxTry=100):
	'''
		In dieser Funktion wird der String zusammengebaut, um mittels "wget" Host, die das "https" Protokoll verweden, zu suchen.
		Ixquick unterstuetzt im Gegensatz zu Google oder Yahoo, nur die Post Methode. 
	'''
	
	i = 0;
	while i < maxTry:
		ixquickUrl = "http://ixquick.com/do/search?"
		j = str(i*10);
		wgetpostCmd = postCmd + j
		wgetPOSTOption = "--post-data"
		outOption = "--output-document=search.%d.%d" %(suffix,i)
		cmds = ["wget", outOption ,wgetPOSTOption, wgetpostCmd, ixquickUrl]
		print cmds
		#time.sleep(random.randint(1,2))
		subprocess.call(cmds)
		i += 1;
	

def parseLink(hostList,path):#
	'''
		In dieser Funktion werden nur die Seiten genommen, die "https" benutzen und in einer Textdatei hinterlegt.
		Es wird nur das "https" Protokoll verwendet, weil diese Server mit sehr grosser Wahrscheinlichkeit ein Zertifikat verweden.
	'''
	file_des = file(path)
	dataLine = file_des.readline()
	
	linkReg = r"((https):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+-=\\\.&]*)"
	
	reg = re.compile(linkReg)
	
	while dataLine:
		m = reg.search(dataLine)
		#print m
		if m:
			#print m.group(0)
			url = m.group(0)[8:]
			#print url
			i = url.find('/')
			if i > 0:
				url = url[0:i]
			hostList.add(url)
				
		dataLine = file_des.readline() 
		
	return hostList

		
if __name__ == "__main__":
	ixPostCmd = [
	"cmd=process_search&language=english&qid=MHLNMSLSNSMK&rcount=&rl=NONE&query=https&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=deutsch&qid=MBLNMSNKMKRT&rcount=&rl=NONE&query=https&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=espanol&qid=LCLNMSNKOTSO&rcount=&rl=NONE&query=https&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=english&qid=LELNMSNRTOTP&rcount=&rl=NONE&query=bank&cat=web&startat=",
	"cmd=process_search&language=francais&qid=LDLNMSNRTMNQ&rcount=&rl=NONE&query=https&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=italiano&qid=MGLNMSNRTNLO&rcount=&rl=NONE&query=https&cat=web&startat=",
	"cmd=process_search&language=nederlands&qid=LHLNMSNRTNPN&rcount=&rl=NONE&query=https&cat=web&startat=",
	"cmd=process_search&language=norsk&qid=LALNMSNRTNTM&rcount=&rl=NONE&query=https&cat=web&startat=",
	"cmd=process_search&language=polski&qid=LFLNMSNRTONK&rcount=&rl=NONE&query=https&cat=web&startat="
	"cmd=process_search&language=english&qid=MHLNMSLSNSMK&rcount=&rl=NONE&query=bank&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=deutsch&qid=MBLNMSNKMKRT&rcount=&rl=NONE&query=bank&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=espanol&qid=LCLNMSNKOTSO&rcount=&rl=NONE&query=bank&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=francais&qid=LDLNMSNRTMNQ&rcount=&rl=NONE&query=bank&cat=web&engine0=v1all&startat=",
	"cmd=process_search&language=italiano&qid=MGLNMSNRTNLO&rcount=&rl=NONE&query=bank&cat=web&startat=",
	"cmd=process_search&language=nederlands&qid=LHLNMSNRTNPN&rcount=&rl=NONE&query=bank&cat=web&startat=",
	"cmd=process_search&language=norsk&qid=LALNMSNRTNTM&rcount=&rl=NONE&query=bank&cat=web&startat=",
	"cmd=process_search&language=polski&qid=LFLNMSNRTONK&rcount=&rl=NONE&query=bank&cat=web&startat="
	"cmd=process_search&language=english&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=login+https&cat=web&engine0=v1all&startat="
	"cmd=process_search&language=deutsch&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=login+https&cat=web&engine0=v1all&startat="
	"cmd=process_search&language=deutsch&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=anmelden+https&cat=web&engine0=v1all&startat="
	"cmd=process_search&language=espano&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=login+https&cat=web&engine0=v1all&startat="
	"cmd=process_search&language=deutsch&qid=MJLNMSQOQRPL&rcount=&rl=NONE&query=shop%2Bhttps&cat=web&startat=",
	"cmd=process_search&language=english&qid=LILNMSQOQSON&rcount=&rl=NONE&query=shop%2Bhttps&cat=web&startat="
	"cmd=process_search&language=deutsch&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=login&bank+https&cat=web&engine0=v1all&startat="
	"cmd=process_search&language=deutsch&qid=LDLNMSPMSMSO&rcount=&rl=NONE&query=bank&ssl+https&cat=web&engine0=v1all&startat="
	
	]
	
	maxTry = 500
	suffix = 0
	for cmd in ixPostCmd:
		genericGetPage(cmd,suffix, maxTry)
	
		hostList = set()
		i = 0;
		while i < maxTry:
			fileName = "search.%d.%d" %(suffix,i)
			print "parse %s ..." %fileName
			parseLink(hostList, fileName)
			i +=1
		suffix += 1
			
		#print hostList
		print "num of host:%d" %len(hostList)
		
		listOfHost = file ("collectListOfHost","a")
		for h in hostList:
			listOfHost.write(h+"\n")
	
	getHost("collectListOfHost")
	
	
	
	
